package jmars.mobile.worker;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;

import jmars.mobile.common.Investigator;
import jmars.mobile.common.Sha;
import jmars.mobile.datainvestigation.Summary;

/**
 * 
 * @author Jan
 *
 */
public class EntropyTrafficTotal implements Investigator {
	private Long computeAll(File[] files,Map<String,Double> entropies) throws IOException{
		Long total = 0L;
		Long line;
		int filesFinished=0;
		FileInputStream fis;
		BufferedReader reader;
		String row;
		String[] cols;
		Map<String,Double> groupCounts=new HashMap<String,Double>();
		Map<String,Map<String,Double>> properties=new HashMap<String,Map<String,Double>>();;
		String groupKey = null,targetKey;
		DecimalFormat df = new DecimalFormat();
		df.applyPattern("0.000");
		Double p;
		Double h;
		for(File inputFile:files){
			fis = new FileInputStream(inputFile);
			reader = new BufferedReader(new InputStreamReader(fis));
			line=0L;
			while((row=reader.readLine())!=null){
				line++;
				try{
					cols=row.split(Summary.columnSeperator);
					if(Summary.filterRow(cols,true,true)) continue;
					if(Summary.parseBeginDate(cols)==null) continue;
//					groupKey="";
//					for(Integer idx:Summary.groupColumnIdx){
//						if(idx<cols.length) {
//							groupKey=groupKey+"$_$"+cols[idx]+"$_$";
//						}
//						else groupKey=groupKey+"$_$$_$";
//					}
					groupKey=Summary.genGroupKey(cols);
					targetKey=Summary.genTargetKey(cols);
					if(properties.containsKey(groupKey)){
						if(properties.get(groupKey).containsKey(targetKey)){
							properties.get(groupKey).put(targetKey, properties.get(groupKey).get(targetKey)+Double.parseDouble(cols[10]));
						}else{
							properties.get(groupKey).put(targetKey, Double.parseDouble(cols[10]));
						}
						groupCounts.put(groupKey, groupCounts.get(groupKey)+Double.parseDouble(cols[10]));
					}else{
						properties.put(groupKey, new HashMap<String,Double>());
						properties.get(groupKey).put(targetKey, Double.parseDouble(cols[10]));
						groupCounts.put(groupKey,Double.parseDouble(cols[10]));
						entropies.put(groupKey, 0D);
						total++;
					}
				}catch(Exception ex){
					System.out.println("exception at line "+line+" of "+inputFile.getName());
					System.out.println(row);
					ex.printStackTrace();
					if(!Summary.errorResume){
						System.exit(0);
					}
				}
			}
			filesFinished++;
			reader.close();
			fis.close();
			System.out.println(filesFinished+": "+inputFile.getName()+" finished at "+line);
		}
		for(String k:properties.keySet()){
			h=0D;
			for(String kk:properties.get(k).keySet()){
				p=properties.get(k).get(kk)/groupCounts.get(k);
				h=h-p*Math.log(p)/Math.log(2);
			}
			entropies.put(k, h);
		}
		return total;
	}
	
	private Long computeBatch(File[] files,Map<String,Double> entropies) throws IOException{
		Long total = 0L;
		FileInputStream fis;
		BufferedReader reader;
		String row;
		String[] cols;
		Map<String,Double> groupCounts=new HashMap<String,Double>();
		Map<String,Map<String,Double>> properties=new HashMap<String,Map<String,Double>>();;
		String groupKey = null,targetKey;
		DecimalFormat df = new DecimalFormat();
		df.applyPattern("0.000");
		Double p;
		Double h;
		for(File inputFile:files){
			fis = new FileInputStream(inputFile);
			reader = new BufferedReader(new InputStreamReader(fis));
			while((row=reader.readLine())!=null){
				cols=row.split(Summary.columnSeperator);
				if(Summary.filterRow(cols,true,true)) 
					continue;
				groupKey=Summary.genGroupKey(cols);
				
				if(!groupCounts.containsKey(groupKey)){
					groupCounts.put(groupKey,0D);
					entropies.put(groupKey, 0D);
				}
			}
			reader.close();
			fis.close();
		}
		Iterator<String> itGks=groupCounts.keySet().iterator();
		while(itGks.hasNext()){
			properties=new HashMap<String,Map<String,Double>>();
			for(int t=1;t<=Summary.batch;t++){
				properties.put(itGks.next(), new HashMap<String,Double>());
				if(!itGks.hasNext())break;
			}
			for(File inputFile:files){
				fis = new FileInputStream(inputFile);
				reader = new BufferedReader(new InputStreamReader(fis));
				while((row=reader.readLine())!=null){
					cols=row.split(Summary.columnSeperator);
					if(Summary.filterRow(cols,true,true)) continue;
					groupKey = Summary.genGroupKey(cols);
					if(properties.containsKey(groupKey)){
						targetKey=Summary.genTargetKey(cols);
						if(properties.get(groupKey).containsKey(targetKey)){
							properties.get(groupKey).put(targetKey, properties.get(groupKey).get(targetKey)+Double.parseDouble(cols[10]));
						}else{
							properties.get(groupKey).put(targetKey, Double.parseDouble(cols[10]));
						}
						groupCounts.put(groupKey, groupCounts.get(groupKey)+Double.parseDouble(cols[10]));
					}
				}
				reader.close();
				fis.close();
			}
			//calculate the entropies for this batch H=-sigma P*log(2,P)
			for(String k:properties.keySet()){
				h=0D;
				for(String kk:properties.get(k).keySet()){
					p=properties.get(k).get(kk)/groupCounts.get(k);
					h=h-p*Math.log(p)/Math.log(2);
				}
				entropies.put(k, h);
			}
			total+=properties.size();
			System.out.println("finished: "+total+" of "+groupCounts.size());
		}
		return total;
	}
	
	@Override
	public Long compute(File[] files, FileWriter writer) throws IOException {
		Long total = 0L;
		Map<String,Double> entropies=new HashMap<String,Double>();
		if(Summary.batch==null)
			total=computeAll(files,entropies);
		else
			total=computeBatch(files,entropies);
		DecimalFormat df = new DecimalFormat();
		df.applyPattern("0.000");
		writer.write("%"+Summary.groupColumns+": "+df.format(entropies.size())+"\r\n");
		writer.write("%total: "+total+"\r\n");
		ArrayList<Entry<String,Double>> l = new ArrayList<Entry<String,Double>>(entropies.entrySet());    

        Collections.sort(l, new Comparator<Map.Entry<String, Double>>() {
            public int compare(Map.Entry<String, Double> o1, Map.Entry<String, Double> o2) {
                return o2.getValue()==o1.getValue()?0:(o2.getValue()>o1.getValue()?1:-1);
            }
        });

		writer.write("%*****************details******************\r\n%");
		for(Integer idx:Summary.groupColumnIdx){
			writer.write(Summary.columns[idx]+",");
		}
		for(Integer idx:Summary.targetColumnIdx){
			writer.write(Summary.columns[idx]+"_");
		}
		writer.write("entropy\r\n");
        for(Entry<String,Double> e : l) {
            //writer.write(e.getKey().replaceAll("\\$_\\$\\$_\\$", ",").replaceAll("\\$_\\$","")+","+e.getValue());
            writer.write(Sha.digest32HexSHA1(e.getKey())+","+e.getValue());
            writer.write("\r\n");
        }
		return total;
	}

}
